// Copyright 2013 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package dfa.axis.v1_19.matchtables;

import com.google.api.ads.common.lib.exception.ValidationException;
import com.google.api.ads.dfa.axis.factory.DfaServices;
import com.google.api.ads.dfa.lib.client.DfaSession;
import com.google.api.client.auth.oauth2.Credential;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.util.Arrays;

/**
* Creates Match Tables using DT file input.
*
* @author Moshe Kahan
*/
public class CreateMatchTables {

  // Prints out a usage message.
  private static void usage() {
    System.err.println(
        "Usage:"
        + "\n\tCreateMatchTables"
        + " <DT Input File #1>"
        + " <DT Input File #2>"
        + " ..."
        + " <DT Input File #N>"
        + " <Match Table output path>");
    System.exit(1);
  }

  /**
   * CreateMatchTables main method. Expects list of DT files and then MT file output 
   * path as command line arguments.
   */
  public static void main(String[] args) {
    TableSettings tableSettings = null;
    try {
      // Read in table settings from properties file.
      tableSettings = new TableSettings("matchtables.properties");
    } catch (IOException e) {
      System.out.println("Error in attempting to read matchtables.properties: " + e);
      e.printStackTrace();
      System.exit(2);
    }

    if (tableSettings.getOauth2ClientId().equals("INSERT_CLIENT_ID_HERE")
        || tableSettings.getOauth2ClientSecret().equals("INSERT_CLIENT_SECRET_HERE")) {
      throw new IllegalArgumentException(
          "Please input your client IDs or secret into the matchtable.properties file. "
          + "See https://code.google.com/apis/console#access");
    } else if (tableSettings.getDfaUserProfileName().equals("INSERT_DFA_USER_PROFILE_NAME_HERE")) {
      throw new IllegalArgumentException(
          "Please input your DFA User Profile name into the matchtable.properties file.");
    }

    // Inputs for DT input file, Match table output path, and field delimiter are required.
    if (args.length < 2) {
      usage();
    }

    // Handle credentialing using OAuth2
    Credential credential = null;
    try {
      credential = new OAuth2CommandLine(
        tableSettings.getOauth2ClientId(),
        tableSettings.getOauth2ClientSecret(),
        tableSettings.getOauth2RefreshToken()).getOAuth2Credential();
    } catch (IOException e) {
      System.out.println("Error in attempting to refresh OAuth2 credentials: " + e);
      e.printStackTrace();
      System.exit(3);
    }

    // Construct a DfaSession.
    DfaSession session = null;
    try {
      session = new DfaSession.Builder()
      .withUsernameAndOAuth2Credential(tableSettings.getDfaUserProfileName(), credential)
      .withEnvironment(DfaSession.Environment.PRODUCTION)
      .build();
    } catch (ValidationException e) {
      System.out.println("Error in attempting to set up DFA SOAP API client session: " + e);
      e.printStackTrace();
      System.exit(4);
    }
    DfaServices dfaServices = new DfaServices();

    // Create LookupObjectManager.
    LookupObjectManager lookupObjectManager =
        new LookupObjectManager(dfaServices, session, tableSettings);

    try {
      String[] dtFiles = Arrays.copyOf(args, args.length - 1);
      String outputPath = args[args.length - 1];
      validateFilesAndFolder(dtFiles, outputPath);
      System.out.println("Parsing DT Files");
      // Parse lines of DT file and send each line for aggregation to lookupObjectManager.

      parseDTFiles(dtFiles, lookupObjectManager);
      System.out.println("Parsing DT Files Done");

      // Generate Match Tables.
      System.out.println("Generating Match Tables");
      lookupObjectManager.generateMatchTables(args[args.length - 1]);
      System.out.println("Generating Match Tables Done");
    } catch (DataTransferFileException e) {
      handleException(e, "Error in attempting to parse Data Transfer files", 5);
    } catch (MatchTableFileException e) {
      handleException(e, "Error in attempting to write to match tables", 6);
    } catch (DfaSoapException e) {
      handleException(e, "Error in attempting to use DFA SOAP API", 7);
    }
  }

  private static void handleException(Exception e, String messge, int exitCode) {
    System.out.print(messge + ": " + e);
    if (e.getCause() != null) {
      System.out.println(" with root cause: " + e.getCause());
      e.getCause().printStackTrace();
    } else {
      System.out.println();
      e.printStackTrace();
    }
    System.exit(exitCode);
  }

  private static void validateFilesAndFolder(String[] files, String folder)
      throws DataTransferFileException, MatchTableFileException {
    for (String file : files) {
      File f = new File(file);
      if (!f.isFile()) {
        throw new DataTransferFileException("Data Transder file: "
            + file + " can not be found.");
      }
    }
    File f = new File(folder);
    if (!f.isDirectory()) {
      throw new MatchTableFileException("Match Table output path: "
          + folder + " can not be found.");
    }
  }

  // Read all DT Files line by line. Each file is sent to parseDTFile.
  private static void parseDTFiles(String[] dtFileNames, LookupObjectManager lookupObjectManager)
      throws DataTransferFileException {
    for (String dtFileName : dtFileNames) {
      lookupObjectManager.clearHeaders();
      parseDTFile(dtFileName, lookupObjectManager);
    }
  }

  // Read DT File line by line.  Use LookupObjectManager to handle each line for each
  // match table we want to generate.
  private static void parseDTFile(String dtFileName, LookupObjectManager lookupObjectManager)
      throws DataTransferFileException {
    final int million = 1000000;
    BufferedInputStream bufferedInputStream = null;
    DtFileScanner scanner = null;
    try {
      System.out.println("Parsing DT File: " + dtFileName);
      bufferedInputStream = new BufferedInputStream(new FileInputStream(dtFileName));
      scanner = new DtFileScanner(bufferedInputStream);
      String line = scanner.headerLine();
      if (line != null && !line.isEmpty()) {
        // Assuming first line is header
        lookupObjectManager.parseHeader(line);
        long x = 0;
        int delimitersNeeded = lookupObjectManager.getHeaderColumns() - 1;
        char delimiter = lookupObjectManager.getRegexDelimiter().charAt(0);
        do {
          try {
            ++x;
            // read in the line
            line = scanner.nextLine(delimiter, delimitersNeeded);
            if (line != null && !line.isEmpty()) {
              // process the line
              lookupObjectManager.parse(line);
            }
          } catch (DataTransferFileException e) {
            // Log bad line and skip
            System.out.println("Possibly corrupt DT file on line #" + x + " with message: "
                + e.getMessage());
          } catch (NumberFormatException e) {
            // Log bad line and skip
            System.out.println("Corrupt DT file line #" + x + " with contents: " + line);
          }
          if (x % million == 0) {
            // Progress in form of [XXX million] records printed every 10mil records
            System.out.print("\r[" + x / million +  " million lines parsed]");
          }
        } while (line != null);
        if (x >= million) {
          System.out.println();
        }
      }
      bufferedInputStream.close();
    } catch (FileNotFoundException e) {
      throw new DataTransferFileException("Data Transfer file not found", e);
    } catch (IOException e) {
      throw new DataTransferFileException("IOException when reading Data Transfer file", e);
    }
  }

  private static class DtFileScanner {
    private InputStream inputStream;
    StringBuilder stringBuilder;

    DtFileScanner(InputStream inputStream) {
      this.inputStream = inputStream;
      stringBuilder = new StringBuilder(1024);
    }

    String headerLine() throws IOException {
      stringBuilder.setLength(0);
      int read = -1;
      while ((read = inputStream.read()) != -1 && read != '\n') {
        stringBuilder.append(((char) read));
      }
      return stringBuilder.toString();
    }

    String nextLine(char delimiter, int delimitersNeeded)
        throws DataTransferFileException, IOException {
      stringBuilder.setLength(0);
      int read = -1;
      int delimiterCount = 0;
      boolean middleColumns = false;

      // \n stops the read unless we have seen some columnDelimiters but not all
      // of the column delimiters we expected. In that case, keep reading even
      // if we read in a line delimiter.
      while ((read = inputStream.read()) != -1 &&
          (middleColumns || read != '\n')) {
        // We only care about Time and numeric ID fields, so this cast from byte
        // to char is fine no matter the encoding of the character data.
        stringBuilder.append(((char) read));
        if (read == delimiter) {
          ++delimiterCount;
          if (delimiterCount < delimitersNeeded) {
            middleColumns = true;
          } else {
            middleColumns = false;
          }
        }
      }

      String line = stringBuilder.toString();
      if (read == -1 && line.isEmpty()) {
        return null;
      }
      if (delimiterCount == 0) {
        // Assume this was an invalid line delimiter in the first column.
        // Log and continue
        String message = "No delimiters found";
        if (line != null && !line.isEmpty()) {
          message = message + " on line: " + line;
        }
        throw new DataTransferFileException(message);
      }
      return line;
    }
  }
}
